import scrapy

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36'
}
cookies = {
    'maxPageNum7603065': '425',
    '__jsluid_s': 'e256909b52702fdde298c334ed7b2d65',
    '__jsl_clearance_s': '1763347121.503|0|wIuwZmDhP4TpQdHEu3TC2PIU3pc%3D'
}


class A公安部Spider(scrapy.Spider):
    name = "公安部"

    async def start(self):
        url = 'https://www.mps.gov.cn/n7598382/index.html'
        yield scrapy.Request(url=url, callback=self.parse, headers=headers, cookies=cookies)

    def parse(self, response):
        list = response.xpath('//ul[@class="list"]/li')
        for item in list:
            date = item.xpath('./span/text()').get()
            link = item.xpath('./a/@href').get()
            title = item.xpath('./a/text()').get()
            print(date, link, title)

        new_list = response.xpath('//div[@style="display:none"]/a')
        for i in range(len(new_list) - 1, -1, -1):
            new_link = new_list[i].xpath('./@href').get()
            new_link = 'https://www.mps.gov.cn' + new_link[2:]
            yield scrapy.Request(url=new_link, callback=self.page_parse, headers=headers, cookies=cookies)

    def page_parse(self, response):
        list = response.xpath('//ul[@class="list"]/li')
        for item in list:
            date = item.xpath('./span/text()').get()
            link = item.xpath('./a/@href').get()
            title = item.xpath('./a/text()').get()
            print(date, link, title)
